import datetime
import re

import requests
import scrapy
from scrapy_redis.spiders import RedisSpider
from scrapy import Request

from myscrapy.items import PostItem

class CnblogsSpider(RedisSpider):
    name = 'cnblogs'
    allowed_domains = ['cnblogs.com']
    start_urls = ['http://cnblogs.com/']  # 开始地址

    def start_requests(self):
        # yield Request('https://www.cnblogs.com/greyzeng/p/16803808.html', dont_filter=True, callback=self.post)
        url = 'https://www.cnblogs.com/sitehome/p/'
        for page in range(1, 200):
            yield Request(url + str(page), dont_filter=True)

    ## 提取分页文章
    def parse(self, response):
        homepageUrls = response.css('.post-item-foot a:first-child::attr(href)').getall()
        homepageUrls = list(set(homepageUrls))
        yield from response.follow_all(homepageUrls, callback=self.homepage)

    ### 主页文章列表
    def homepage(self, response):
        ## 阅读全文
        urls = response.css('.c_b_p_desc_readmore::attr(href)').getall()
        for value in urls:
            ## https://www.cnblogs.com/xiaoxiongcanguan/p/16879296.html
            desc = ''
            ret = re.search(r"(\d+).html", value)
            if ret:
                desc_id = '#postlist_description_'+ret.group(1)
                desc = ''.join(response.css(desc_id+"::text").getall())
            yield scrapy.Request(value, callback=self.post, meta={"desc": desc})

        ### 下一页
        next1 = response.css('#nav_next_page a:first-child::attr(href)').get()
        next2 = response.css('.pager a:last-child::attr(href)').get()
        next = next1 or next2
        print("下一页", next)
        if next:
            yield scrapy.Request(next, callback=self.homepage)

    def getDesc(self, id, response):
        desc_id_select = "#postlist_description_{id}"
        divselect = desc_id_select.format(id=id)
        desc_img = response.css(divselect + ' .desc_img::text').get()
        return desc_img or response.css(divselect + "::text").get()

    ### 获取文章内容
    def post(self, response):
        cb_entryId = 0
        cb_blogId = 0

        enIdMatch = re.search('cb_entryId = (\d+),', response.text)
        if enIdMatch:
            cb_entryId = enIdMatch.group(1)

        cbIdMatch = re.search('cb_blogId = (\d+),', response.text)
        if cbIdMatch:
            cb_blogId = cbIdMatch.group(1)

        title = re.search("cb_postTitle = '(.*)';", response.text)
        title = title.group(1)

        desc = response.meta["desc"]
        tags = []
        if cb_entryId and cb_blogId:
            url = "https://www.cnblogs.com/weskynet/ajax/CategoriesTags.aspx?blogId={blogId}&postId={postId}"
            catehtml = requests.get(url.format(blogId=cb_blogId, postId=cb_entryId))
            tags = re.findall('\">(.*?)<', catehtml.text)

        item = PostItem()
        item['url'] = response.url
        item['desc'] = desc
        item['title'] = title
        item['tags'] = list(set(tags))
        item['author'] = response.css('#Header1_HeaderTitle::text').get()
        item['content'] = ''.join(response.css('#cnblogs_post_body *').getall())
        item['created_at'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        yield item

    def getPostDesc(self, id, response):
        text = response.css('#postlist_description_{id}'.format(id=id)).get()
        desc_img = re.search('class="desc_img">(\s\S+)', text)
        desc = response.css('#postlist_description_{id}::text'.format(id=id)).get()
        if desc_img:
            desc = desc_img.group(0).replace('class="desc_img">', '')
        return desc
